from fake_useragent import UserAgent
import requests
import chardet
from lxml import html
from Utils.RedisHelper import RedisHelper
import time
import json
import random


def getua():
    ua = UserAgent()
    return ua.__getattr__('random')


def GetHtml(url, keyword=''):
    n = 0
    htmlcontent = ''
    while htmlcontent == '':
        n = n + 1
        if n > 3:
            return ''
        try:
            proxy = GetRandomProxy()
            if n == 1:
                r = requests.get(url, headers={'User-Agent': getua()}, timeout=10, verify=False)
            else:
                r = requests.get(url, headers={'User-Agent': getua()}, timeout=10, proxies=proxy, verify=False)
            if r.status_code != 200:
                htmlcontent = ''
            else:
                htmlcontent = r.content.strip()
                if htmlcontent == '':
                    continue
                char = chardet.detect(htmlcontent)
                htmlcontent = htmlcontent.decode(char['encoding'])
                if keyword.strip() != "":
                    if htmlcontent.find(keyword) > -1:
                        return htmlcontent
                    else:
                        return ""
        except Exception:
            htmlcontent = ''
            continue
        return htmlcontent


def GetDataByXpath(htmlcontent,config):
    htmldoc = html.etree.HTML(htmlcontent)
    ips = htmldoc.xpath(config['ip'])
    ports = htmldoc.xpath(config['port'])
    if len(config['protocol'])>0:
        protocols = htmldoc.xpath(config['protocol'])
    if len(config['port'])>0:
        ports = htmldoc.xpath(config['port'])

    datacount = len(ips)-1
    maxindex = datacount-1
    dic = []
    for index in range(0, maxindex):
        data = {}
        if len(config['protocol'])>0:
            data['protocol'] = protocols[index]
        else:
            data['protocol'] = 'http'
        if len(config['port'])>0:
            data['port'] = ports[index]
        data['ip'] = ips[index]
        dic.append(data)
    return dic


def CheckProxy(key):
    redis = RedisHelper()
    data = redis.get(key)
    t = time.time()
    code, proxy = ValidProxy(data['protocol'], data['ip'], data['port'])
    timespan = time.time() - t
    if code:
        data['ts'] = timespan
        data['time'] = time.time()
        redis.set(key, data)
    else:
        redis.delete(key)


def CheckTempProxy(key):
    redis = RedisHelper()
    data = redis.temp_get(key)
    t = time.time()
    code, proxy = ValidProxy(data['protocol'], data['ip'], data['port'])
    timespan = time.time() - t
    if code:
        data['ts'] = timespan
        data['time'] = time.time()
        redis.set(key, data)
    redis.temp_delete(key)


def ValidProxy(protocol, ip, port, web='http://httpbin.org/ip'):
    proxy =  'http://{}:{}'.format(ip, port)
    proxies = {'http': proxy, 'https': proxy}
    try:
        r = requests.get('http://httpbin.org/ip', proxies=proxies, timeout=5, verify=False)
        if r.status_code == 200:
            webip = json.loads(r.text)['origin']
            if webip == ip:
                return True, proxy
            else:
                return False, 'transparent' + proxy
        else:
            return False, 'not 200:'+proxy
    except Exception:
        return False, 'error:'+proxy


def GetRandomProxy():
    redis = RedisHelper()
    keys = redis.getkeys()
    if len(keys) > 0:
        key = random.choice(keys)
        data = redis.get(key)
        proxyip = 'http://{0}:{1}'.format(data['ip'], data['port'])
        proxy = {'http': proxyip, 'https': proxyip}
        return proxy
    else:
        return {}
